{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob\n",
    "import zipfile\n",
    "\n",
    "## utility\n",
    "\n",
    "# 定义转换函数\n",
    "def format_stock_code(df, column_name):\n",
    "    \"\"\"\n",
    "    股票代码转换成6位,并添加交易所名称\n",
    "    \"\"\"\n",
    "    df[column_name] = df[column_name].astype(str)\n",
    "    df[column_name] = df[column_name].apply(lambda x: x.zfill(6))\n",
    "    # 根据股票代码的开头进行后缀的添加\n",
    "    df.loc[df[column_name].str.startswith(('0', '3')), column_name] += '.SZ'\n",
    "    df.loc[df[column_name].str.startswith(('4', '8')), column_name] += '.BJ'\n",
    "    df.loc[df[column_name].str.startswith('6'), column_name] += '.SH'\n",
    "    return df\n",
    "\n",
    "def read_and_concatenate_csv(folder_path,columns):\n",
    "    \"\"\"\n",
    "    使用glob模块获取文件夹下所有CSV文件的路径\n",
    "    \"\"\"\n",
    "    csv_files = glob.glob(folder_path + '*.csv')\n",
    "    \n",
    "    # 创建一个空列表，用于存储每个CSV文件的DataFrame\n",
    "    dfs = []\n",
    "    \n",
    "    # 逐个读取CSV文件\n",
    "    for file in csv_files:\n",
    "        df = pd.read_csv(file)\n",
    "        df = df[columns]\n",
    "        dfs.append(df)\n",
    "        \n",
    "    # 使用concat方法合并所有DataFrame\n",
    "    concatenated_df = pd.concat(dfs, ignore_index=True)\n",
    "    \n",
    "    return concatenated_df\n",
    "\n",
    "def get_oth_data(folder_path,columns=None):\n",
    "\n",
    "    ## 导入文件夹所有csv\n",
    "    df = read_and_concatenate_csv(folder_path,columns)\n",
    "    ## 修改股票代码\n",
    "    df = format_stock_code(df, '股票代码_Stkcd')\n",
    "    ## 日期标准格式\n",
    "    df['日期_Date'] = pd.to_datetime(df['日期_Date'] )\n",
    "    ## 按code和date排序\n",
    "    df = df.sort_values(by=['股票代码_Stkcd','日期_Date'])\n",
    "    df = df[columns] \n",
    "\n",
    "    return df\n",
    "\n",
    "def get_OHLC(file_path):\n",
    "\n",
    "    ## 获取数据\n",
    "    df = pd.read_csv(file_path,encoding='gbk')\n",
    "    keywords = [\"开盘价\", \"最高价\", \"最低价\", \"收盘价\"]\n",
    "    keywords_1 = [\"Open\", \"High\", \"Low\", \"Close\"]\n",
    "\n",
    "    # 找到含关键字的行的索引\n",
    "    indices = [i for i, val in enumerate(df.iloc[:,0]) if val in keywords]\n",
    "    # 添加最后一个索引+1，以便能够捕获到最后一个关键字后的所有行\n",
    "    indices.append(len(df))\n",
    "    \n",
    "    # 根据关键字分割DataFrame\n",
    "    dfs = [df.iloc[indices[n]:indices[n+1]-2] for n in range(len(indices)-1)]\n",
    "\n",
    "    ## 构造标准格式\n",
    "    y = []\n",
    "    for ind,x in enumerate(dfs):\n",
    "        ## 删除、修改列\n",
    "        x = x.drop(columns=['Unnamed: 0','Unnamed: 1','日期'])\n",
    "        x = x.rename(columns={'Date':'code'})\n",
    "        ## 宽格式转换为长格式\n",
    "        x= pd.melt(x, id_vars=['code'], var_name='date', value_name=keywords_1[ind])\n",
    "        ## 时间标准格式\n",
    "        x['date'] = pd.to_datetime(x['date'])\n",
    "        # 设置双索引\n",
    "        x.set_index(['date', 'code'], inplace=True)\n",
    "        x = x.sort_index(level='date')\n",
    "        ## 转换成数值格式\n",
    "        x = x.apply(pd.to_numeric, errors='coerce')\n",
    "        y.append(x)\n",
    "\n",
    "    return  pd.concat(y,axis=1)\n",
    "\n",
    "def get_data():\n",
    "    \n",
    "    pass    \n",
    "\n",
    "def get_preprocess():\n",
    "\n",
    "    ##\n",
    "    pass\n",
    "\n",
    "## 29指标名称\n",
    "columns = ['股票代码_Stkcd','日期_Date','上市状态_Listedstate','证监会行业门类代码_Csrciccd1',\n",
    "           '成交量_Trdvol','成交金额_Trdsum','日振幅(%)_Dampltd','总股数日换手率(%)_DFulTurnR',\n",
    "           '流通股日换手率(%)_DTrdTurnR','累积股价调整乘子_Mcfacpr','汇率_Ex','日收益率_Dret',\n",
    "           '等权平均市场日收益率_Dreteq','流通市值加权平均市场日收益率_Drettmv','总市值加权平均市场日收益率_Dretmc',\n",
    "           '等权平均市场日资本收益率_Dareteq','流通市值加权平均市场日资本收益_Darettmv','总市值加权平均日资本收益_Daretmc',\n",
    "           '市盈率_PE','市净率_PB','市现率_PCF','市销率_PS','每股收益(摊薄)(元/股)_EPS','净资产收益率(摊薄)_ROE',\n",
    "           '每股公积金(元/股)_AccumFundPS','每股营业利润(元/股)_OpPrfPS','每股净资产(元/股)_NAPS',\n",
    "           '每股营业收入_IncomePS','每股经营活动现金流量净额(元/股)_NCFfropePS','已上市流通股_Lsttrdshr'\n",
    "           ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Lincoln\\AppData\\Local\\Temp\\ipykernel_28388\\2091867672.py:32: DtypeWarning: Columns (25) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv(file)\n",
      "C:\\Users\\Lincoln\\AppData\\Local\\Temp\\ipykernel_28388\\2091867672.py:58: DtypeWarning: Columns (0,1,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000,1001,1002,1003,1004,1005,1006,1007,1008,1009,1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072,1073,1074,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1085,1086,1087,1088,1089,1090,1091,1092,1093,1094,1095,1096,1097,1098,1099,1100,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1111,1112,1113,1114,1115,1116,1117,1118,1119,1120,1121,1122,1123,1124,1125,1126,1127,1128,1129,1130,1131,1132,1133,1134,1135,1136,1137,1138,1139,1140,1141,1142,1143,1144,1145,1146,1147,1148,1149,1150,1151,1152,1153,1154,1155,1156,1157,1158,1159,1160,1161,1162,1163,1164,1165,1166,1167,1168,1169,1170,1171,1172,1173,1174,1175,1176,1177,1178,1179,1180,1181,1182,1183,1184,1185,1186,1187,1188,1189,1190,1191,1192,1193,1194,1195,1196,1197,1198,1199,1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211,1212,1213,1214,1215,1216,1217,1218,1219,1220,1221,1222,1223,1224,1225,1226,1227,1228,1229,1230,1231,1232,1233,1234,1235,1236,1237,1238,1239,1240,1241,1242,1243,1244,1245,1246,1247,1248,1249,1250,1251,1252,1253,1254,1255,1256,1257,1258,1259,1260,1261,1262,1263,1264,1265,1266,1267,1268,1269,1270,1271,1272,1273,1274,1275,1276,1277,1278,1279,1280,1281,1282,1283,1284,1285,1286,1287,1288,1289,1290,1291,1292,1293,1294,1295,1296,1297,1298,1299,1300,1301,1302,1303,1304,1305,1306,1307,1308,1309,1310,1311,1312,1313,1314,1315,1316,1317,1318,1319,1320,1321,1322,1323,1324,1325,1326,1327,1328,1329,1330,1331,1332,1333,1334,1335,1336,1337,1338,1339,1340,1341,1342,1343,1344,1345,1346,1347,1348,1349,1350,1351,1352,1353,1354,1355,1356,1357,1358,1359,1360,1361,1362,1363,1364,1365,1366,1367,1368,1369,1370,1371,1372,1373,1374,1375,1376,1377,1378,1379,1380,1381,1382,1383,1384,1385,1386,1387,1388,1389,1390,1391,1392,1393,1394,1395,1396,1397,1398,1399,1400,1401,1402,1403,1404,1405,1406,1407,1408,1409,1410,1411,1412,1413,1414,1415,1416,1417,1418,1419,1420,1421,1422,1423,1424,1425,1426,1427,1428,1429,1430,1431,1432,1433,1434,1435,1436,1437,1438,1439,1440,1441,1442,1443,1444,1445,1446,1447,1448,1449,1450,1451,1452,1453,1454,1455,1456,1457,1458,1459,1460,1461,1462,1463,1464,1465,1466,1467,1468,1469,1470,1471,1472,1473,1474,1475,1476,1477,1478,1479,1480,1481,1482,1483,1484,1485,1486,1487,1488,1489,1490,1491,1492,1493,1494,1495,1496,1497,1498,1499,1500,1501,1502,1503,1504,1505,1506,1507,1508,1509) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df = pd.read_csv(file_path,encoding='gbk')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>上市状态_Listedstate</th>\n",
       "      <th>证监会行业门类代码_Csrciccd1</th>\n",
       "      <th>成交量_Trdvol</th>\n",
       "      <th>成交金额_Trdsum</th>\n",
       "      <th>日振幅(%)_Dampltd</th>\n",
       "      <th>总股数日换手率(%)_DFulTurnR</th>\n",
       "      <th>流通股日换手率(%)_DTrdTurnR</th>\n",
       "      <th>累积股价调整乘子_Mcfacpr</th>\n",
       "      <th>汇率_Ex</th>\n",
       "      <th>...</th>\n",
       "      <th>每股公积金(元/股)_AccumFundPS</th>\n",
       "      <th>每股营业利润(元/股)_OpPrfPS</th>\n",
       "      <th>每股净资产(元/股)_NAPS</th>\n",
       "      <th>每股营业收入_IncomePS</th>\n",
       "      <th>每股经营活动现金流量净额(元/股)_NCFfropePS</th>\n",
       "      <th>已上市流通股_Lsttrdshr</th>\n",
       "      <th>Open</th>\n",
       "      <th>High</th>\n",
       "      <th>Low</th>\n",
       "      <th>Close</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th>code</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2021-02-01</th>\n",
       "      <th>000001.SZ</th>\n",
       "      <td>20</td>\n",
       "      <td>Norm</td>\n",
       "      <td>J</td>\n",
       "      <td>147523930.0</td>\n",
       "      <td>3.529557e+09</td>\n",
       "      <td>9.9177</td>\n",
       "      <td>0.7602</td>\n",
       "      <td>0.7602</td>\n",
       "      <td>122.9998</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.72</td>\n",
       "      <td>0.58</td>\n",
       "      <td>14.88</td>\n",
       "      <td>1.97</td>\n",
       "      <td>1.58</td>\n",
       "      <td>1.940576e+10</td>\n",
       "      <td>21.8838</td>\n",
       "      <td>23.7772</td>\n",
       "      <td>21.5984</td>\n",
       "      <td>23.3586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000002.SZ</th>\n",
       "      <td>748</td>\n",
       "      <td>Norm</td>\n",
       "      <td>K</td>\n",
       "      <td>88669362.0</td>\n",
       "      <td>2.456622e+09</td>\n",
       "      <td>3.0946</td>\n",
       "      <td>0.9118</td>\n",
       "      <td>0.9118</td>\n",
       "      <td>119.2797</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>9.21</td>\n",
       "      <td>4.33</td>\n",
       "      <td>17.45</td>\n",
       "      <td>24.83</td>\n",
       "      <td>2.87</td>\n",
       "      <td>9.724197e+09</td>\n",
       "      <td>23.6902</td>\n",
       "      <td>23.8173</td>\n",
       "      <td>23.0889</td>\n",
       "      <td>23.7749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000004.SZ</th>\n",
       "      <td>1477</td>\n",
       "      <td>Norm</td>\n",
       "      <td>I</td>\n",
       "      <td>4688700.0</td>\n",
       "      <td>9.048045e+07</td>\n",
       "      <td>5.6269</td>\n",
       "      <td>2.8407</td>\n",
       "      <td>4.0725</td>\n",
       "      <td>4.5505</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>7.35</td>\n",
       "      <td>0.35</td>\n",
       "      <td>8.78</td>\n",
       "      <td>0.57</td>\n",
       "      <td>-0.11</td>\n",
       "      <td>1.151316e+08</td>\n",
       "      <td>19.9600</td>\n",
       "      <td>19.9600</td>\n",
       "      <td>18.8200</td>\n",
       "      <td>19.1400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000005.SZ</th>\n",
       "      <td>2205</td>\n",
       "      <td>Norm</td>\n",
       "      <td>N</td>\n",
       "      <td>19355012.0</td>\n",
       "      <td>3.788038e+07</td>\n",
       "      <td>7.4766</td>\n",
       "      <td>1.8285</td>\n",
       "      <td>1.8295</td>\n",
       "      <td>5.9172</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.79</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1.53</td>\n",
       "      <td>0.08</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>1.057946e+09</td>\n",
       "      <td>2.0800</td>\n",
       "      <td>2.0900</td>\n",
       "      <td>1.9300</td>\n",
       "      <td>1.9400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000006.SZ</th>\n",
       "      <td>2933</td>\n",
       "      <td>Norm</td>\n",
       "      <td>K</td>\n",
       "      <td>4760431.0</td>\n",
       "      <td>2.327165e+07</td>\n",
       "      <td>2.2634</td>\n",
       "      <td>0.3526</td>\n",
       "      <td>0.3526</td>\n",
       "      <td>55.3750</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.69</td>\n",
       "      <td>0.47</td>\n",
       "      <td>5.19</td>\n",
       "      <td>1.19</td>\n",
       "      <td>-0.03</td>\n",
       "      <td>1.349995e+09</td>\n",
       "      <td>4.4554</td>\n",
       "      <td>4.5473</td>\n",
       "      <td>4.4463</td>\n",
       "      <td>4.5289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">2023-12-29</th>\n",
       "      <th>873679.BJ</th>\n",
       "      <td>3266587</td>\n",
       "      <td>Norm</td>\n",
       "      <td>C</td>\n",
       "      <td>1615423.0</td>\n",
       "      <td>3.762497e+07</td>\n",
       "      <td>3.3391</td>\n",
       "      <td>2.8924</td>\n",
       "      <td>12.6422</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.98</td>\n",
       "      <td>1.23</td>\n",
       "      <td>7.00</td>\n",
       "      <td>3.15</td>\n",
       "      <td>1.68</td>\n",
       "      <td>1.277800e+07</td>\n",
       "      <td>22.9700</td>\n",
       "      <td>23.5900</td>\n",
       "      <td>22.8200</td>\n",
       "      <td>23.4800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>873693.BJ</th>\n",
       "      <td>3266632</td>\n",
       "      <td>Norm</td>\n",
       "      <td>C</td>\n",
       "      <td>1421837.0</td>\n",
       "      <td>5.952405e+07</td>\n",
       "      <td>4.7470</td>\n",
       "      <td>1.9558</td>\n",
       "      <td>7.0041</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.56</td>\n",
       "      <td>0.29</td>\n",
       "      <td>4.39</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.21</td>\n",
       "      <td>2.030000e+07</td>\n",
       "      <td>42.0600</td>\n",
       "      <td>42.9800</td>\n",
       "      <td>41.0100</td>\n",
       "      <td>41.5400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>873703.BJ</th>\n",
       "      <td>3266650</td>\n",
       "      <td>Norm</td>\n",
       "      <td>C</td>\n",
       "      <td>2031030.0</td>\n",
       "      <td>7.836494e+07</td>\n",
       "      <td>2.7146</td>\n",
       "      <td>2.7207</td>\n",
       "      <td>14.2528</td>\n",
       "      <td>1.0193</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.39</td>\n",
       "      <td>1.63</td>\n",
       "      <td>5.06</td>\n",
       "      <td>5.08</td>\n",
       "      <td>1.82</td>\n",
       "      <td>1.425000e+07</td>\n",
       "      <td>38.4900</td>\n",
       "      <td>39.1600</td>\n",
       "      <td>38.1100</td>\n",
       "      <td>38.4800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>873726.BJ</th>\n",
       "      <td>3266701</td>\n",
       "      <td>Norm</td>\n",
       "      <td>C</td>\n",
       "      <td>4661968.0</td>\n",
       "      <td>2.004090e+08</td>\n",
       "      <td>25.5395</td>\n",
       "      <td>5.6800</td>\n",
       "      <td>26.7293</td>\n",
       "      <td>1.0187</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.87</td>\n",
       "      <td>0.99</td>\n",
       "      <td>4.58</td>\n",
       "      <td>2.81</td>\n",
       "      <td>2.00</td>\n",
       "      <td>1.744142e+07</td>\n",
       "      <td>39.4300</td>\n",
       "      <td>47.9600</td>\n",
       "      <td>37.9000</td>\n",
       "      <td>43.6400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>873833.BJ</th>\n",
       "      <td>3266745</td>\n",
       "      <td>Norm</td>\n",
       "      <td>C</td>\n",
       "      <td>2603982.0</td>\n",
       "      <td>3.875303e+07</td>\n",
       "      <td>3.9136</td>\n",
       "      <td>3.1617</td>\n",
       "      <td>8.9071</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3.27</td>\n",
       "      <td>0.80</td>\n",
       "      <td>6.46</td>\n",
       "      <td>4.94</td>\n",
       "      <td>1.09</td>\n",
       "      <td>2.923500e+07</td>\n",
       "      <td>14.8600</td>\n",
       "      <td>15.1100</td>\n",
       "      <td>14.5300</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3184342 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                        index 上市状态_Listedstate 证监会行业门类代码_Csrciccd1  \\\n",
       "date       code                                                      \n",
       "2021-02-01 000001.SZ       20             Norm                   J   \n",
       "           000002.SZ      748             Norm                   K   \n",
       "           000004.SZ     1477             Norm                   I   \n",
       "           000005.SZ     2205             Norm                   N   \n",
       "           000006.SZ     2933             Norm                   K   \n",
       "...                       ...              ...                 ...   \n",
       "2023-12-29 873679.BJ  3266587             Norm                   C   \n",
       "           873693.BJ  3266632             Norm                   C   \n",
       "           873703.BJ  3266650             Norm                   C   \n",
       "           873726.BJ  3266701             Norm                   C   \n",
       "           873833.BJ  3266745             Norm                   C   \n",
       "\n",
       "                       成交量_Trdvol   成交金额_Trdsum  日振幅(%)_Dampltd  \\\n",
       "date       code                                                   \n",
       "2021-02-01 000001.SZ  147523930.0  3.529557e+09          9.9177   \n",
       "           000002.SZ   88669362.0  2.456622e+09          3.0946   \n",
       "           000004.SZ    4688700.0  9.048045e+07          5.6269   \n",
       "           000005.SZ   19355012.0  3.788038e+07          7.4766   \n",
       "           000006.SZ    4760431.0  2.327165e+07          2.2634   \n",
       "...                           ...           ...             ...   \n",
       "2023-12-29 873679.BJ    1615423.0  3.762497e+07          3.3391   \n",
       "           873693.BJ    1421837.0  5.952405e+07          4.7470   \n",
       "           873703.BJ    2031030.0  7.836494e+07          2.7146   \n",
       "           873726.BJ    4661968.0  2.004090e+08         25.5395   \n",
       "           873833.BJ    2603982.0  3.875303e+07          3.9136   \n",
       "\n",
       "                      总股数日换手率(%)_DFulTurnR  流通股日换手率(%)_DTrdTurnR  \\\n",
       "date       code                                                    \n",
       "2021-02-01 000001.SZ                0.7602                0.7602   \n",
       "           000002.SZ                0.9118                0.9118   \n",
       "           000004.SZ                2.8407                4.0725   \n",
       "           000005.SZ                1.8285                1.8295   \n",
       "           000006.SZ                0.3526                0.3526   \n",
       "...                                    ...                   ...   \n",
       "2023-12-29 873679.BJ                2.8924               12.6422   \n",
       "           873693.BJ                1.9558                7.0041   \n",
       "           873703.BJ                2.7207               14.2528   \n",
       "           873726.BJ                5.6800               26.7293   \n",
       "           873833.BJ                3.1617                8.9071   \n",
       "\n",
       "                      累积股价调整乘子_Mcfacpr  汇率_Ex  ...  每股公积金(元/股)_AccumFundPS  \\\n",
       "date       code                                ...                           \n",
       "2021-02-01 000001.SZ          122.9998    1.0  ...                    4.72   \n",
       "           000002.SZ          119.2797    1.0  ...                    9.21   \n",
       "           000004.SZ            4.5505    1.0  ...                    7.35   \n",
       "           000005.SZ            5.9172    1.0  ...                    0.79   \n",
       "           000006.SZ           55.3750    1.0  ...                    1.69   \n",
       "...                                ...    ...  ...                     ...   \n",
       "2023-12-29 873679.BJ            1.0000    1.0  ...                    0.98   \n",
       "           873693.BJ            1.0000    1.0  ...                    1.56   \n",
       "           873703.BJ            1.0193    1.0  ...                    1.39   \n",
       "           873726.BJ            1.0187    1.0  ...                    1.87   \n",
       "           873833.BJ            1.0000    1.0  ...                    3.27   \n",
       "\n",
       "                      每股营业利润(元/股)_OpPrfPS  每股净资产(元/股)_NAPS  每股营业收入_IncomePS  \\\n",
       "date       code                                                               \n",
       "2021-02-01 000001.SZ                 0.58            14.88             1.97   \n",
       "           000002.SZ                 4.33            17.45            24.83   \n",
       "           000004.SZ                 0.35             8.78             0.57   \n",
       "           000005.SZ                 0.01             1.53             0.08   \n",
       "           000006.SZ                 0.47             5.19             1.19   \n",
       "...                                   ...              ...              ...   \n",
       "2023-12-29 873679.BJ                 1.23             7.00             3.15   \n",
       "           873693.BJ                 0.29             4.39             0.70   \n",
       "           873703.BJ                 1.63             5.06             5.08   \n",
       "           873726.BJ                 0.99             4.58             2.81   \n",
       "           873833.BJ                 0.80             6.46             4.94   \n",
       "\n",
       "                      每股经营活动现金流量净额(元/股)_NCFfropePS  已上市流通股_Lsttrdshr     Open  \\\n",
       "date       code                                                                 \n",
       "2021-02-01 000001.SZ                          1.58      1.940576e+10  21.8838   \n",
       "           000002.SZ                          2.87      9.724197e+09  23.6902   \n",
       "           000004.SZ                         -0.11      1.151316e+08  19.9600   \n",
       "           000005.SZ                         -0.01      1.057946e+09   2.0800   \n",
       "           000006.SZ                         -0.03      1.349995e+09   4.4554   \n",
       "...                                            ...               ...      ...   \n",
       "2023-12-29 873679.BJ                          1.68      1.277800e+07  22.9700   \n",
       "           873693.BJ                          0.21      2.030000e+07  42.0600   \n",
       "           873703.BJ                          1.82      1.425000e+07  38.4900   \n",
       "           873726.BJ                          2.00      1.744142e+07  39.4300   \n",
       "           873833.BJ                          1.09      2.923500e+07  14.8600   \n",
       "\n",
       "                         High      Low    Close  \n",
       "date       code                                  \n",
       "2021-02-01 000001.SZ  23.7772  21.5984  23.3586  \n",
       "           000002.SZ  23.8173  23.0889  23.7749  \n",
       "           000004.SZ  19.9600  18.8200  19.1400  \n",
       "           000005.SZ   2.0900   1.9300   1.9400  \n",
       "           000006.SZ   4.5473   4.4463   4.5289  \n",
       "...                       ...      ...      ...  \n",
       "2023-12-29 873679.BJ  23.5900  22.8200  23.4800  \n",
       "           873693.BJ  42.9800  41.0100  41.5400  \n",
       "           873703.BJ  39.1600  38.1100  38.4800  \n",
       "           873726.BJ  47.9600  37.9000  43.6400  \n",
       "           873833.BJ  15.1100  14.5300      NaN  \n",
       "\n",
       "[3184342 rows x 33 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df =  get_oth_data(folder_path='../量化data/csv_data/', columns=columns)\n",
    "df = df.rename(columns={'股票代码_Stkcd':'code','日期_Date':'date'})\n",
    "dfs = get_OHLC('../量化data/(open,high,low,close).csv')\n",
    "df_ohlc = dfs.reset_index()\n",
    "merged_df = pd.merge(df, df_ohlc, on=['code', 'date'], how='left')\n",
    "merged_df.reset_index(inplace = True)\n",
    "condition = (merged_df['date'] >= '2021-02-01') & (merged_df['date'] < '2024-03-01') #时间选取2021-04-01至2023-09-30\n",
    "data = merged_df[condition]\n",
    "data.set_index(['date', 'code'], inplace=True)\n",
    "data= data.sort_index(level='date')\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv('../量化data/股票整合数据_未预处理.zip', compression='zip', index=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13 (main, Aug 25 2022, 23:51:50) [MSC v.1916 64 bit (AMD64)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "f52f256adf2cc158a51e888fcb9b6688be269c3160efd3d6e246b7ec509c991c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
